/***************************************************************************************************
 *Copyright(C),2010-2016,Sumscope
 *FileName	:  DictTable.cpp
 *Author	:  scofined.qi
 *Version	:  1.0
 *Date		:  2017/04/01
 *Desc		:  二进制映射文件形式的新字典类
 *Relation :
 *Others	:  //其他内容说明
 *Function :  //主要函数列表，每条记录应包含函数名及功能简要说明
 *History	:
 * //修改历史记录列表，每条修改记录应包含修改日期、修改者及修改内容简介
 ***************************************************************************************************/
/*
        总共有kDictTypeCount个字典,每个字典对应一个二进制映射文件

        此二进制映射文件的格式如下:
                首先是TableHead结构体,然后是若干个字段的描述结构的数组,类型是FieldDesc
                再就是等长大小的记录
        每个字典的具体字段描述在DictTableCreator.cpp中
                SetPredictSize用于进度条预估进度
                SetDebugKey用于根据配置文件在特定key输出日志
                SetTag用于设定该文件是否加密
*/
// #include <stdlib.h>
#include "DictTable.h"

#include <core/strings/Rand.h>
#include <core/strings/gbkmap.h>
#include <core/system/system_util.h>

#include "DictExport.h"
#include "DictLoadListener.h"
#include "HTable/HTable.h"
#include "HTable/src/ITableCache.h"
#include "MacroIndexDict.h"
// #include "DictType.h"

// #include "..\SSTools/ConfigVM.h"
#include <core/FileSys.h>
#include <core/pointer/scoped_pointer.h>
#include <core/strings/FormatTable.h>
#include <core/strings/base64.h>
#include <core/strings/string_util.h>

#include <QCoreApplication>
#include <QDir>
#include <QStandardPaths>
#include <QTemporaryFile>

using namespace qb::base;
using namespace qb::ht;

using qb::base::CGbkMap;
using qb::ht::RecordResult;

#ifdef WIN32
#else
#include <alloca.h>
#endif

#ifndef MEM_METRIC
#define MEM_METRIC  //
#endif
#ifndef BVIEW
#define BVIEW  //
#endif
#ifndef BS_CFGMETRICS
#define BS_CFGMETRICS  //
#endif
#ifndef BS_METRICS
#define BS_METRICS  //
#endif

#define FIELD_OFFSET(type, field) ((long)(long)&(((type*)0)->field))
#define RTL_FIELD_SIZE(type, field) (sizeof(((type*)0)->field))
#define MAX_PATH 260

namespace qb {
namespace ht {
#define METRIC_ADDTIME(INDEX)  // metric.AddTime(INDEX)
//#define CHECK_FIELD_POS(TYPE,FIELD,POS,SIZE) \
//					static_assert(FIELD_OFFSET(TYPE, FIELD)==POS && RTL_FIELD_SIZE(TYPE, FIELD)==SIZE,"CHECK_FIELD_POS "#TYPE"."#FIELD);

//#define CHECK_FIELD_SIZE(DESTTYPE,FIELD1,SRCTYPE,FIELD2) \
//					static_assert(RTL_FIELD_SIZE(DESTTYPE, FIELD1)==RTL_FIELD_SIZE(SRCTYPE, FIELD2),#DESTTYPE"."#FIELD1"!="#SRCTYPE"."#FIELD2);\

#define COPY_FIELD(DESTTYPE, DEST, FIELD1, SRCTYPE, SRC, FIELD2) \
  memcpy(DEST.FIELD1, SRC.FIELD2, RTL_FIELD_SIZE(SRCTYPE, FIELD2));

void CopyDoubleToField(char* field, int size, const char* fmt, double dvalue) {
  int ncopy = FMTBUFN(field, size, fmt, dvalue);
#ifdef _DEBUG
  if (ncopy < size) memset(field + ncopy, 0, size - ncopy);
#endif
}
bool IsAsciiString(const unsigned char* str, int len) {
  const unsigned char* end = str + len;
  do {
    if (*str > 127) return false;
    str++;
  } while (str < end);
  return true;
}

void CopyStringToField(char* dest, int capcity, const char* source, int len) {
  // dest内存已经全部置零
  // 参数无需检查
  len = len >= capcity ? (capcity - 1) : len;
  if (len > 0) memcpy(dest, source, len);
#ifdef _DEBUG
  if (len < capcity) memset(dest + len, 0, capcity - len);
#endif
}
void CopyUtf8ToField(char* dest, int capcity, CConvertFast& convert,
                     const char* utf8, int len) {
  /*
  碰到的大多数utf8如果只是ascii字符,那么无需转化
  拉全量测试结果表明
  所有字段中,出现纯ascii字符串的概率为 93.07%,
  6.93%的中文字符转码成gbk耗费时间占据了总时间的58.15%
  总体转化434w个字符串,从1696036us降低到467127us,性能提升了263%
  */
  if (IsAsciiString((const unsigned char*)utf8, len)) {
    CopyStringToField(dest, capcity, utf8, len);
  } else {
    const char* source = convert.ToGbk(utf8, len);
    CopyStringToField(dest, capcity, source, convert.GetStrSize());
  }
}
void StringLinkInt(char* buf, int cap, const char* str, int num) {
  char* ptr = buf;
  char* end = buf + cap;
  for (; ptr < end && *str != 0;) *ptr++ = *str++;
  *ptr++ = '|';
#ifdef WIN32
  _itoa(num, ptr, 10);
#else
  sprintf(ptr, "%d", num);
#endif
}
bool IsDictFileWritable(const BaseString& fname) {
  if (FileSys::instance()->isFileExist(fname.c_str()))
    return FileSys::instance()->isFileWritable(fname.c_str());

  QFile* file;
  bool bRet = FileSys::instance()->createFile(fname.c_str(),
                                              QIODevice::ReadWrite, file);
  FileSys::instance()->closeFile(file);
  return bRet;
}

void MakeTempPathFileName(const char* name, std::string& tempfile) {
  QString strFileName =
      QDir::tempPath() + QDir::separator() + QString("QB") + QString(name);
  QTemporaryFile tmpFile(strFileName);
  tmpFile.open();
  tmpFile.close();
  tempfile = strFileName.toStdString();
}
bool MakeSysPathFileName(const char* name, std::string& tempfile) {
  QString AppDataLocation =
      QStandardPaths::writableLocation(QStandardPaths::AppDataLocation);
  // \\Qeubee\\Bonddata\\%s
  BaseString path = AppDataLocation.toStdString().c_str();
  path += BaseString("/Qeubee/Bonddata/");
  FileSys::instance()->makePathExist(path.c_str());
  path += BaseString(name);
  tempfile = path;
  return true;
}
void PrintDebugRecord(const char* key, const char* record, int keyoffset,
                      int flagoffset, int id) {
  if (key && strcmp(key, record + keyoffset) == 0) {
    // const ST_BOND_LIST& bl = *((const BOND_LIST*)record);
    log_debug("[DICT][DEBUGKEY]flag:%d key:%s index:%d ", record[flagoffset],
              record + keyoffset, id);
  }
}
void GbkStringReplace(char* gbk, int size, char src, char dest) {
  for (int i = 0; i < size; ++i) {
    if (gbk[i] == src)
      gbk[i] = dest;
    else if (gbk[i] < 0 || gbk[i] > 127)
      ++i;
  }
}
void LogInvalidField(const FieldDesc& fd, const char* table, const char* cur) {
  log_error("[DICT]invalid cur:%s kDataType:%s  %s.%s", cur,
            GetDataTypeName(fd.type), table, fd.name);
}
namespace {
void LoadSequenceSet(std::set<int>& seqset, const char* fname) {
  std::vector<BaseString> lines;
  StringUtil::LoadLinesFromFile(G2W(fname), lines);
  for (int i = 0; i < (int)lines.size(); i++)
    seqset.insert(atoi(lines[i].c_str()));
}
int CreateDiffSequnceSet(std::set<int>& seqset, const char* fname1,
                         const char* fname2) {
  std::set<int> s1, s2;
  LoadSequenceSet(s1, fname1);
  LoadSequenceSet(s2, fname2);
  seqset.clear();
  for (auto it = s1.begin(); it != s1.end(); ++it) {
    if (s2.find(*it) == s2.end()) seqset.insert(*it);
  }
  int nsize = seqset.size();
  log_info("[DICT]find %6d in %s not in %s", nsize, fname1, fname2);
  for (auto it = s2.begin(); it != s2.end(); ++it) {
    if (s1.find(*it) == s1.end()) seqset.insert(*it);
  }
  log_info("[DICT]find %6d in %s not in %s", seqset.size() - nsize, fname2,
           fname1);
  return seqset.size();
}

}  // namespace
CBufferFile::CBufferFile(const char* fname, int capcity) {
  FileSys::instance()->createFile(fname, QIODevice::ReadWrite, m_file);
  m_capcity = capcity;
  m_size = 0;
  m_buffer = new char[capcity];
  memset(m_buffer, 0, capcity);
}

CBufferFile::~CBufferFile() {
  if (m_file != NULL) {
    if (m_buffer && m_size > 0) {
      m_file->write(m_buffer, m_size);
    }
    FileSys::instance()->closeFile(m_file);
  }
  if (m_buffer) delete[] m_buffer;
}

void CBufferFile::AddLine(const char* str, int len) {
  if (m_file == NULL || str == NULL) return;
  if (m_size + len + 2 > m_capcity) {
    m_file->write(m_buffer, m_size);
    memset(m_buffer, 0, m_capcity);
    m_size = 0;
  }
  memcpy(m_buffer + m_size, str, len);
  m_size += len;
  memcpy(m_buffer + m_size, "\r\n", 2);
  m_size += 2;
}

//////////////////////////////////////////////////////////////////////////
DictTable::DictTable(const DictDesc& dd)
    : m_type(dd.GetType()),
      m_fname(dd.m_fname),
      m_key_column(dd.m_key_column),
      m_flag_column(dd.m_flag_column),
      m_table(NULL),
      m_viewsize(dd.GetViewSize()),
      m_increase_size(dd.GetIncreaseSize()),
      m_keyfield_maker(NULL),
      m_pod_record(NULL),
      m_pod_record_size(0),
      m_key_linkchar('.'),
      m_macroindex_dict(NULL) {
  m_seq_col = m_seq_offset = -1;
  if (dd.GetType() == kDictTypeMacroIndex)
    m_macroindex_dict = new MacroIndexDict();

  SetKeyColumn(dd.m_key_column);
  SetFlagColumn(dd.m_flag_column);

  // 用于针对哪个字典的解码包进行日志输出,慎用,会造成大量日志和性能卡顿
  static_assert(kDictTypeCount <= 31, "kDictTypeCount must be smaller than 31");
  m_dict_flag.reset();
  unsigned int index = 1;  // 2^0
  unsigned int flag =
      0;  // qb::base::cfg::GetConfigInt(kProfileDictTypeCheckFlag);
  for (int i = 0; i < kDictTypeCount; i++) {
    m_dict_flag.set(i, index & flag);
    index = index << 1;
  }

  m_record_descs.assign(dd.m_fields.begin(), dd.m_fields.end());
  if (dd.m_keyfields.size() > 0)
    m_key_split_fields.assign(dd.m_keyfields.begin(), dd.m_keyfields.end());
  m_keyfield_maker = dd.m_keymaker;

  memset(&m_head, 0, sizeof(TableHead));
  m_head.tag = TAG;
  FIELDCOPY(m_head.name, dd.m_name.c_str());
  FIELDCOPY(m_head.version, dd.m_version.c_str());
  SetRecordName(dd.m_record_name);
  SetRecordSize(dd.m_record_size);
  // 初始化文件头部的余下字段
  BASSERT(m_head.record_size % 4 == 0);
  int headsize = sizeof(TableHead) + m_record_descs.size() * sizeof(FieldDesc);
  m_head.crc = 0;
  m_head.head_size = headsize;
  m_head.desc_offset = sizeof(TableHead);
  m_head.record_offset = ROUND_ALIGN(headsize, m_head.record_size);
  m_head.desc_item_size = sizeof(FieldDesc);
  m_head.desc_item_count = m_record_descs.size();
  m_head.record_count = 0;
  m_head.key_column = dd.m_key_column;
  m_head.flag_column = dd.m_flag_column;
  m_head.del_count = 0;
  m_head.view_size = dd.GetViewSize();
  m_head.inc_size = dd.GetIncreaseSize();

  if (m_type == kDictTypeBondList && m_dict_flag.test(m_type)) {
    CreateDiffSequnceSet(m_diffset, "./Bonddata/SequenceClient.txt",
                         "./Bonddata/SequenceServer.txt");
  }
}
DictTable::~DictTable() {
  m_update_map.Reset(nullptr);
  if (m_pod_record) delete[] m_pod_record;
  if (m_table) delete m_table;
  m_table = NULL;

  // m_keyfield_maker 是全局对象DictDesc中的指针,这里只借用,不能删除
  // if (m_keyfield_maker)
  //	delete m_keyfield_maker;
  // m_keyfield_maker = NULL;

  if (m_macroindex_dict) delete m_macroindex_dict;
  m_macroindex_dict = NULL;
}
int DictTable::GetColumnByName(const char* name) const {
  for (int i = 0; i < (int)m_record_descs.size(); i++) {
    const FieldDesc& fd = m_record_descs[i];
    if (strcmp(fd.name, name) == 0) return i;
  }
  return -1;
}
void DictTable::SetKeySplitColumns(vector<FieldDesc>& fields) {
  /*
  此调用应该在所有AddRecordInfo之后
  此函数用来设置关键列的拼装列

  对于某些数据如BONDINFO,
  typedef	struct BONDINFO{
  char BondKey[35+1];				//债券key B0000332012CORLEB01
  char ListedMarket[3+1];			//市场代码 CIB/SSE/SZE
  ...
  char Flag;
  //不写入文件,D:删除 U:更新 I:插入 }BONDINFO;
  将其存储在DictTable中,要求必须有一个关键列
  那么,外部使用时将原先的BondKey和ListedMarket两列放在一起,作为一列当作关键列
  此函数用来指示关键列实际包含的数据列定义,这些定义,用于从更新包中接收原始数据
  如果原始数据结构如BONDINFO中就有关键列,那么此函数可以不调用
  这里的断言都不能忽略
  */
  BASSERT(m_key_column >= 0 && m_key_column < (int)m_record_descs.size());
  BASSERT(fields.size() > 0);
  FieldDesc& keyfd = m_record_descs[m_key_column];
  BASSERT(keyfd.offset == fields[0].offset);
  m_key_split_fields.clear();

  int size_sum = fields[0].size;
  for (int i = 1; i < (int)fields.size(); i++) {
    FieldDesc& fd = fields[i];
    BASSERT(fd.type ==
            (kDataType)kDataTypeCharArray);  // 要求只能是固定长度字符串类型
    BASSERT(size_sum + keyfd.offset == fd.offset);
    size_sum += fd.size;
  }
  BASSERT(size_sum == keyfd.size);
  m_key_split_fields.assign(fields.begin(), fields.end());
}

void DictTable::SetRecordName(const std::string& record) {
  FIELDCOPY(m_head.record_name, record.c_str());
}

void DictTable::AddRecordInfo(int type, int offset, int size, const char* name,
                              const char* desc) {
  AddRecordInfo(FieldDesc((kDataType)type, offset, size, name, desc));
}
void DictTable::AddRecordInfo(const FieldDesc& fd) {
  BASSERT(fd.offset >= 0 && fd.size >= 1 && fd.name && fd.desc);
  BASSERT(fd.type > kDataTypeNone && fd.type < kDataTypeCount);
  m_record_descs.push_back(fd);
}
bool DictTable::CheckLoadParams(string& error) {
  // 参数检查,正式版本不应该出现此类错误
  // 出现此类错误大都是错误使用导致
  if (m_record_descs.size() <= 0 || m_head.record_size <= 0 ||
      m_key_column < 0 || m_flag_column < 0 ||
      m_key_column >= (int)m_record_descs.size() ||
      m_flag_column >= (int)m_record_descs.size() ||
      m_key_column == m_flag_column) {
    error = "DictTable 没有正确初始化";
    return false;
  }
  FieldDesc keyfd = m_record_descs[m_key_column];
  FieldDesc flagfd = m_record_descs[m_flag_column];
  if (keyfd.type != kDataTypeCharArray || flagfd.size != 1) {
    error = "错误的关键列或者标记列";
    return false;
  }
  return true;
}
bool DictTable::Load(string& error) {
  if (!CheckLoadParams(error)) {
    BASSERT(0);
    return false;
  }

  // 初始化文件头部的余下字段
  BASSERT(m_head.record_size % 4 == 0);
  int headsize = m_head.head_size;
  // 创建HTable类
  TableDesc td;
  for (int i = 0; i < (int)m_record_descs.size(); i++) {
    FieldDesc& fd = m_record_descs[i];
    td.AddColumn(fd.name, fd.desc, (kDataType)fd.type, (kDataType)fd.type,
                 fd.offset, fd.size);
  }
  td.SetKeyColumn((qb::ht::kColumn)m_key_column);
  td.SetFlagColumn((qb::ht::kColumn)m_flag_column);

  // 如果指定文件不可写
  // 尝试两次,第二次,直接使用系统临时目录,确保文件可写
  bool tempfile =
      false;  // 如果为true,那么QB退出会删除,字典文件创建时会附带temp标志
  for (int i = 0; i < 2; i++) {
    error = "";
    if (i == 0) {
      if (!IsDictFileWritable(m_fname.c_str())) {
        MakeWritablePath(m_head.name, m_fname, tempfile);
      }
    } else
      MakeWritablePath(m_head.name, m_fname, tempfile);

    // setlocale
    bool fileexist = FileSys::instance()->isFileExist(m_fname.c_str());
    m_table = new qb::ht::Table(td, 0, headsize);
    m_table->SetViewSize(m_viewsize);
    m_table->SetIncreaseSize(m_increase_size);
    m_table->SetName(m_head.name);
    if (!m_table->Initialize(
            m_fname, DictDesc::GetStockDictDesc(m_type).GetTag(), tempfile)) {
      error = "字典文件创建失败";
      if (i == 0 && !tempfile) {
        if (m_table) {
          delete m_table;
        }
        m_table = NULL;
        continue;
      }
      // 严重错误,尽量避免
      log_error("[DICT]第%d此尝试创建字典文件:%s 失败", i + 1, m_head.name);
      return false;
    }
    int length = m_table->GetRecordLength();
    const char* pod = GetPodRecord(length);
    if (!fileexist) {
      UpdateHead();
    }
    return CheckAndLoad(error);
  }
  BASSERT(0);
  return false;
}
void DictTable::MakeWritablePath(const char* name, std::string& fpath,
                                 bool& tempfile) {
  if (!MakeSysPathFileName(m_head.name, m_fname)) {
    QString strFileName = QDir::tempPath() + QDir::separator() + QString("QB") +
                          QString(m_head.name);
    QTemporaryFile tmpFile(strFileName);
    tmpFile.open();
    tmpFile.close();

    // 检查m_fname文件内容
    // 将m_fname文件内容拷贝到temppath中
    std::string error;
    if (CheckFileHeadDirectly(m_fname, error)) {
      QFile file(m_fname.c_str());
      file.copy(strFileName);
    }
    if (error.size() > 0) {
      log_error("[DICT]读取旧字典文件:%s 失败,原因:%s 改为使用临时文件方案",
                StringUtil::GetShortFileName(m_fname.c_str(), 2),
                error.c_str());
    }
    m_fname = strFileName.toStdString();
    tempfile = true;
    log_warning("[DICT]使用临时文件:%s\t%s", m_head.name, m_fname.c_str());
  } else {
    log_warning("[DICT]使用系统数据目录:%s\t%s", m_head.name, m_fname.c_str());
  }
}

void DictTable::Unload() {
  BASSERT(m_table);
  m_table->Unload();
  if (m_type && m_macroindex_dict) m_macroindex_dict;
}
bool DictTable::Reload() {
  BASSERT(m_table);
  m_table->Reload();
  // string error;
  // bool suc= CheckAndLoad(error);
  return true;
}
bool DictTable::GetSequenceInfo(int& col, int& offset) {
  if (m_seq_col >= 0 && m_seq_offset >= 0) {
    col = m_seq_col;
    offset = m_seq_offset;
    return true;
  }
  int seqoffset = -1;
  int seqcol = GetColumnByName("Sequence");
  if (seqcol >= 0) {
    const FieldDesc& fd = m_record_descs[seqcol];
    seqoffset = fd.offset;
  }
  if (seqcol < 0) return false;
  m_seq_col = seqcol;
  m_seq_offset = seqoffset;
  col = seqcol;
  offset = seqoffset;
  return true;
}

namespace {
BaseString FieldToStr(const char* data, int size, kDataType type) {
  char buf[128] = {0};
  switch (type) {
    case kDataTypeDate:
      FMTBUF(buf, "%d", *((int*)data));
      break;
    case kDataTypeFloat:
      FMTBUF(buf, "%f", *((float*)data));
      break;
    case kDataTypeDouble:
      FMTBUF(buf, "%f", *((double*)data));
      break;
    case kDataTypeBool:
      FMTBUF(buf, "%s", (*((bool*)data)) ? "true" : "false");
      break;
    case kDataTypeInt:
      FMTBUF(buf, "%d", *((int*)data));
      break;
#ifdef WIN32
    case kDataTypeInt64:
      FMTBUF(buf, "%I64d", *((int64_t*)data));
      break;
#else
    case kDataTypeInt64:
      FMTBUF(buf, "%lld", *((int64_t*)data));
      break;
#endif
    case kDataTypeChar:
      FMTBUF(buf, "%d", *((char*)data));
      break;
    case kDataTypeCharArray:
      FMTBUF(buf, "%s", data);
      break;
    default:
      break;
  }
  return buf;
}
}  // namespace
void DictDesc::PrintRecord(int index, const char* record, int length,
                           bool removed) const {
  BaseString str, value;
  str = os::fmtstr("%6d:%s==>", index, removed ? "DEL" : "ADD");
  for (int i = 0; i < (int)m_fields.size(); i++) {
    const FieldDesc& fd = m_fields[i];
    value = FieldToStr(record + fd.offset, fd.size, fd.type);
    str += os::fmtstr("%s:%s,", fd.name, value.c_str());
  }
  log_info("[DICT][DATA]%s", str.c_str());
}
void DictDesc::CompareRecord(int i1, const char* r1, int i2, const char* r2,
                             int length) const {
  qb::base::FormatTable ft;
  BaseString v1, v2;
  log_info("[DICT]比较字典%s 的第%6d条记录 和 %6d条记录", m_version.c_str(), i1,
           i2);
  for (int i = 0; i < (int)m_fields.size(); i++) {
    const FieldDesc& fd = m_fields[i];
    v1 = FieldToStr(r1 + fd.offset, fd.size, fd.type);
    v2 = FieldToStr(r2 + fd.offset, fd.size, fd.type);
    ft.AddField("field", fd.name);
    ft.AddField("first", v1);
    ft.AddField("last", v2);
    ft.AddField("comp", v1 == v2 ? "     " : "*****");
  }
  ft.FormatToLog("[DICT][CHECK][COMP]");
}
bool DictTable::GetSequences(vector<int>& sequences) {
  int seqoffset = -1;
  int seqcol = -1;
  int rdlength = GetRecordLength();
  if (!GetSequenceInfo(seqcol, seqoffset) || seqoffset < 0 ||
      seqoffset >= rdlength) {
    return false;
  }

  int flagoffset = m_table->GetColumnInfo(m_flag_column).GetOffset();
  int ncount = m_table->GetMaxRecordId() + 1;
  sequences.reserve(ncount);
  std::map<int, int> indexes;  // seq->index,用于检测出现重复seq的情况
  char* first = nullptr;
  char* last = nullptr;
  int nerror = 0;
  bool bcheck = m_dict_flag.test(m_type);
  for (int i = 0; i < ncount; i++) {
    const char* record = ReadRecord(i);
    if (record == 0) continue;
    if (record[flagoffset]) continue;
    int seq = *((const int*)(record + seqoffset));
    if (seq != 0) {
      if (bcheck)  // 当检测开关开启后,将相同seq的两条记录打印出来
      {
        auto it = indexes.find(seq);
        if (it != indexes.end()) {
          if (!last) last = new char[rdlength + 1];
          memcpy(last, record, rdlength);
          last[rdlength] = 0;
          if (!first) first = new char[rdlength + 1];
          const char* temp = ReadRecord(it->second);
          memcpy(first, temp, rdlength);
          first[rdlength] = 0;
          const DictDesc& dd = DictDesc::GetStockDictDesc(m_type);
          dd.CompareRecord(it->second, first, i, last, rdlength);
          nerror++;
        } else
          indexes[seq] = i;
      }
      sequences.push_back(seq);
    }
  }
  if (first) delete[] first;
  if (last) delete[] last;
  if (sequences.size() > 0) std::sort(sequences.begin(), sequences.end());
  return true;
}
bool DictTable::Export(const char* fullFileName) {
  // 导出格式为cod格式
  DictExport cod(8 * 1024 * 1024,
                 DictDesc::GetStockDictDesc(m_type).GetDoublePrecision(),
                 DictDesc::GetStockDictDesc(m_type).GetTag());
  if (!cod.Init(fullFileName)) {
    log_error("[DICT]Export %s failed", fullFileName);
    return false;
  }

  std::vector<FieldDesc> fields;
  GetExportFields(fields);

  // 针对文业20180403发出的邮件的需求，针对特殊到处字段进行纠正
  for (int i = 0; i < (int)fields.size(); i++) {
    FieldDesc& fd = fields[i];
    if (m_type == kDictTypeBondPrimaryInfo ||
        m_type == kDictTypeBondAdditionInfo) {
      if (strcmp(fd.name, "Auction_Time_Start") == 0 ||
          strcmp(fd.name, "Auction_Time_End") == 0)
        cod.AddConverter(fd.name, fd.offset, DictExport::kConvertTypeBlank);
    } else if (m_type == kDictTypeBondList) {
      if (strcmp(fd.name, "IsMortgage") == 0 ||
          strcmp(fd.name, "IsCrossMarket") == 0 ||
          strcmp(fd.name, "IsMunicipal") == 0)
        cod.AddConverter(fd.name, fd.offset, DictExport::kConvertTypeYesNo);
    }
  }

  if (0) {
    BVIEW("[DICT]表格%s导出列如下:%d", GetName(), fields.size());
    for (int i = 0; i < (int)fields.size(); i++) {
      FieldDesc& fd = fields[i];
      BVIEW("[DICT]%2d\t%s\t%4d-%3d  type:%d", i, fd.name, fd.offset, fd.size,
            fd.type);
    }
  }

  // 将记录按照key的递增排列顺序导出
  int ncount = m_table->GetMaxRecordId() + 1;
  int flagoffset = m_table->GetColumnInfo(m_flag_column).GetOffset();
  int keyoffset = m_table->GetColumnInfo(m_key_column).GetOffset();

  std::vector<int> sequences;
  int seqoffset = -1;
  int seqcol = -1;
  if (GetSequenceInfo(seqcol, seqoffset)) {
    sequences.reserve(ncount);
  }

  class RecordIndexMap {  // 使得kDictTypeMacroIndex字典导出按照旧字典的顺序
   public:
    struct KyeIndex {
      string key;
      int date;

     public:
      bool operator<(const KyeIndex& ki) const {
        if (key == ki.key) return date < ki.date;
        return key < ki.key;
      }
    };
    typedef std::map<KyeIndex, int> KeyIndexMap;
    KeyIndexMap m_indexes;
    void AddIndex(const char* key, int date, int index) {
      KyeIndex ki;
      ki.key = key;
      ki.date = date;
      m_indexes[ki] = index;
    }
  };
  typedef map<string, int> StringIntMap;
  RecordIndexMap keys;
  int rdlength = GetRecordLength();
  const char* dictkey = DictDesc::GetDebugKey();
  for (int i = 0; i < ncount; i++) {
    const char* record = ReadRecord(i);
    if (record == 0) continue;
    PrintDebugRecord(dictkey, record, keyoffset, flagoffset, i);
    if (m_type != kDictTypeMacroIndex && record[flagoffset]) continue;
    if (record && seqoffset >= 0 && seqoffset < rdlength) {
      int seq = *((const int*)(record + seqoffset));
      if (seq != 0) sequences.push_back(seq);
    }
    const char* key = record + keyoffset;
    if (m_type == kDictTypeMacroIndex) {
      const DICT_MACRO_INDEX* pmi = (const DICT_MACRO_INDEX*)record;
      keys.AddIndex(pmi->Code, pmi->Date, i);
    } else
      keys.AddIndex(key, 0, i);
  };
  log_info("[DICT]export table:%s ncount:%d  %d  %d  keys:%d", GetName(),
           ncount, m_table->GetRecordCount(),
           m_macroindex_dict ? m_macroindex_dict->GetCount() : 0,
           keys.m_indexes.size());

  int podsize = rdlength;
  char* pod = m_key_split_fields.size() <= 0 ? 0 : GetPodRecord(podsize);
  for (RecordIndexMap::KeyIndexMap::iterator it = keys.m_indexes.begin();
       it != keys.m_indexes.end(); ++it) {
    int index = it->second;
    const char* cur_record = ReadRecord(index);
    if (pod) {
      memcpy(pod, cur_record, rdlength);
      pod[rdlength] = 0;
      SplitKeyField(pod, rdlength);
    }
    const char* record = pod ? pod : cur_record;
    PrintDebugRecord(dictkey, record, keyoffset, flagoffset, index);
    for (int j = 0; j < (int)fields.size(); j++) {
      FieldDesc& fd = fields[j];
      cod.AddData(fd.type, fd.offset, record + fd.offset, fd.size);
    }
    cod.AddLineEnd();
  }
  if (sequences.size() > 0) {
    string seqname = fullFileName;
    seqname += ".seq.txt";
    CBufferFile bufferfile(seqname.c_str(), 1024 * 1024);
    char buffer[128] = {0};
    for (int i = 0; i < (int)sequences.size(); i++) {
      int len = FMTBUF(buffer, "%d", sequences[i]);
      bufferfile.AddLine(buffer, len);
    }
  }

  return true;
}
bool DictTable::CheckAndLoad(string& error) {
  MEM_METRIC("CheckAndLoad:%s", m_head.name);
  if (!CheckFileHead(error)) {
    log_error("[DICT]检查数据%s失败,原因:%s,后续操作:重建拉全量", m_head.name,
              error.c_str());
    return false;
  }
  if (m_head.del_count > 0) {
    if (!ClearDeleteRecords()) {
      log_error("ClearDeleteRecords failed");
      return false;
    }
  }
  // 各种检查通过后,从映射文件中获取文件头,逐个加载记录,建立索引
  if (m_type == kDictTypeMacroIndex) {
    // 到此时,文件头部已经核查正确无误,需要进一步核查每条MacroIndex记录
    int len = GetRecordLength();
    int ncount = GetMaxRecordId() + 1;
    int pod_size = GetRecordLength();
    int nrecord_error = 0;
    int ndate_error = 0;
    char* pod_record = GetPodRecord(pod_size);
    for (int i = 0; i < ncount; i++) {
      const char* record = ReadRecord(i);
      if (record == NULL) {
        nrecord_error++;
        continue;
      }
      memcpy(pod_record, record, len);
      if (!m_macroindex_dict->UpdateElement(*(DICT_MACRO_INDEX*)pod_record))
        ndate_error++;

      if (nrecord_error > 50 || ndate_error > 20)  // 避免写超级多的错误日志
        break;
    }

    // 核查MacroIndex字典文件,发现错误,需要重置状态,之后会导致该字典重新拉取
    if (nrecord_error > 0 || ndate_error > 0) {
      m_head.updateid = 0;
      memset(m_head.state, 0, sizeof(tName64));
      log_error(
          "[DICT]加载MacroIndex字典存在错误:nrecord_error:%8d,ndate_error:%8d,"
          "ncount:%8d",
          nrecord_error, ndate_error, ncount);
      error = os::fmtstr("MacroIndex字典文件损坏");

      // 重置m_macroindex_dict,使得其不被损坏的字典文件数据污染
      if (m_macroindex_dict) delete m_macroindex_dict;
      m_macroindex_dict = new MacroIndexDict();
      return false;
    }
    m_macroindex_dict->UpdateState(m_head.state, m_head.updateid);
  } else {
    BS_CFGMETRICS(true, "[DICT]表:%s 创建索引  %d条 ", m_head.name,
                  m_head.record_count);
    int keyoffset = m_table->GetColumnInfo(m_key_column).GetOffset();
    int flagoffset = m_table->GetColumnInfo(m_flag_column).GetOffset();
    m_table->GetCache()->BuildKeyMap(m_head.record_count, keyoffset,
                                     flagoffset);
  }
  return true;
}
void DictTable::RecreateFile() { m_table->GetCache()->RecreateFile(); }
bool DictTable::DeleteRecords(double percent, std::string& report) {
  string name = DictDesc::GetStockDictDesc(m_type).GetName();
  if (m_type == kDictTypeMacroIndex) {
    report = os::fmtstr("字典[%s]不支持删除测试", name.c_str());
    return false;
  }

  // 第一步,提取Sequence不为0的记录的索引
  int keyoffset = m_table->GetColumnInfo(m_key_column).GetOffset();
  int flagoffset = m_table->GetColumnInfo(m_flag_column).GetOffset();
  std::vector<int> seq_indexes, after_indexes;
  std::vector<int> del_indexes;
  set<int> addset;
  map<int, int> sequences, after_sequences;

  int seqoffset = -1;
  int seqcol = -1;
  if (!GetSequenceInfo(seqcol, seqoffset)) {
    report = os::fmtstr("字典[%s]中不包含Sequence字段信息", name.c_str());
    return false;
  }

  int ntotal = m_head.record_count;
  for (int i = 0; i < m_head.record_count; i++) {
    const char* record = m_table->ReadRecordDirect(i);
    if (record == NULL) {
      break;
    }
    int seq = *((const int*)(record + seqoffset));
    if (!record[flagoffset] && seq > 0) {
      seq_indexes.push_back(i);
      sequences.insert(map<int, int>::value_type(i, seq));
    }
  }

  // 第二步,根据percent,挑选其中的哪些索引需要删除
  if (percent >= 1.0) {
    del_indexes.assign(seq_indexes.begin(), seq_indexes.end());
    for (int i = 0; i < (int)seq_indexes.size(); i++)
      addset.insert(seq_indexes[i]);
  } else if (percent > 0.0) {
    int npercent = percent * 100;
    int nexpect = seq_indexes.size() * percent + 0.5;
    while ((int)addset.size() < nexpect) {
      for (int i = 0; i < (int)seq_indexes.size(); i++) {
        if (Rand::RandInt(0, 100) <= npercent && (int)addset.size() < nexpect)
          addset.insert(seq_indexes[i]);
      }
    }
    del_indexes.assign(addset.begin(), addset.end());
    std::sort(del_indexes.begin(), del_indexes.end());
  }

  if (seq_indexes.size() <= 0) {
    report = os::fmtstr("字典[%s]中不包含Sequence字段的记录", name.c_str());
    return true;
  }
  if (del_indexes.size() <= 0) {
    report = os::fmtstr(
        "比例设置过小:%.4f,没有删除任何记录,字典[%s]共有%d条有Sequence的记录",
        percent, name.c_str(), seq_indexes.size());
    return true;
  }

  // 第三步,将记录末尾的覆盖需要删除的
  int select = 0;
  int podsize = m_table->GetRecordLength();
  int length = m_table->GetRecordLength();
  int valid_count = m_head.record_count - del_indexes.size();
  char* pod = GetPodRecord(podsize);

  // 将后续有效的记录找出来
  for (int i = valid_count; i < m_head.record_count; i++) {
    if (addset.find(i) != addset.end()) {
      memset(pod, 0, podsize);
      m_table->WriteRecordDirect(i, pod, length);
      continue;
    }
    const char* record = m_table->ReadRecordDirect(i);
    if (record == NULL) {
      report = os::fmtstr("字典[%s]读取出现错误", name.c_str());
      return false;
    }
    int rid = del_indexes[select++];
    BASSERT(rid < valid_count);
    memset(pod, 0, podsize);
    memcpy(pod, record, length);
    m_table->WriteRecordDirect(rid, pod, length);
    memset(pod, 0, podsize);
    m_table->WriteRecordDirect(i, pod, length);
    // printf("\n Move %d to %d", i, rid);
  }
  m_head.record_count = valid_count;
  // m_table->GetCache()->BuildKeyMap(m_head.record_count, keyoffset,
  // flagoffset);
  UpdateHead();

  for (int i = 0; i < m_head.record_count; i++) {
    const char* record = m_table->ReadRecordDirect(i);
    if (record == NULL) {
      break;
    }
    int seq = *((const int*)(record + seqoffset));
    if (!record[flagoffset] && seq > 0) {
      after_indexes.push_back(i);
      after_sequences.insert(map<int, int>::value_type(i, seq));
    }
  }

  // 第四部,生成报告字符串
  report = os::fmtstr(
      "字典[%s]包含%d条记录,%d条有Sequence,删除比例:%.4f,删除个数:%d,"
      "余下记录个数:%d",
      name.c_str(), ntotal, seq_indexes.size(), percent, del_indexes.size(),
      valid_count);
  return true;
}
bool DictTable::ClearDeleteRecords() {
  // 原始文件中存在删除记录,需要将记录删除掉
  // 从末尾将有效记录逐个覆盖掉前面的无效记录所占据的rid
  BS_METRICS("[DICT]");
  int flagoffset = m_table->GetColumnInfo(m_flag_column).GetOffset();
  vector<int> del_indexes;
  del_indexes.reserve(m_head.del_count);
  bool haserror = false;
  int error_index = -1;
  for (int i = 0; i < m_head.record_count; i++) {
    const char* record = m_table->ReadRecordDirect(i);
    if (record == NULL) {
      haserror = true;
      error_index = i;
      break;
    }
    if (record[flagoffset]) del_indexes.push_back(i);
  }
  if (haserror) {
    log_error("invalid record index:%d", error_index);
    return false;
  }

  if (del_indexes.size() != m_head.del_count) {
    log_warning("[DICT]文件头%s 有%d条删除记录,实际检查核实有%d条删除记录",
                m_head.name, m_head.del_count, del_indexes.size());
  }

  int length = m_table->GetRecordLength();
  int valid_count = m_head.record_count - del_indexes.size();
  log_info("待删除记录个数:%d 有效个数:%d 总个数:%d", del_indexes.size(),
           valid_count, m_head.record_count);
  // if (del_indexes.size() > 1024)
  //{
  //	int select = 0;
  //	int podsize = length;
  //	char* pod = GetPodRecord(podsize);
  //	for (int i = valid_count; i < m_head.record_count; i++)
  //	{
  //		const char* record = m_table->ReadRecordDirect(i);
  //		if (record[flagoffset])
  //			continue;
  //		int rid = del_indexes[select++];
  //		memset(pod, 0, podsize);
  //		memcpy(pod, record, length);
  //		m_table->WriteRecordDirect(rid, pod, length);
  //	}
  // }else
  if (del_indexes.size() > 0)  // 少量性能更佳,写的时候是顺序写入
  {
    class RecordVec : public std::vector<char*> {
     public:
      RecordVec(int cap) { this->reserve(cap); }
      ~RecordVec() {
        for (int i = 0; i < (int)size(); i++) {
          if (at(i)) delete[] at(i);
        }
        this->clear();
      }
    };
    RecordVec records(del_indexes.size());
    // 将rid大于有效个数的非删除记录,覆盖前面删除记录所占据的索引位置
    bool alloc_failed = false, read_failed = false;
    for (int i = valid_count; i < m_head.record_count; i++) {
      const char* record = m_table->ReadRecordDirect(i);
      if (record == NULL) {
        read_failed = true;
        break;
      }
      if (record[flagoffset]) continue;
      char* pod = new (std::nothrow) char[length];
      if (pod == 0) {
        alloc_failed = true;
        break;
      }
      memcpy(pod, record, length);
      records.push_back(pod);
    }
    if (alloc_failed || read_failed) {
      log_error("分配内存失败,单个长度:%d,已经分配个数:%d alloc:%s read:%s",
                length, records.size(), alloc_failed ? "true" : "false",
                read_failed ? "true" : "false");
      return false;
    }
    // BASSERTS(del_indexes.size() == records.size(),
    // "[DICT]del_indexes%d!=records%d",del_indexes.size(),records.size());
    for (int i = 0; i < (int)records.size(); i++) {
      if (i >= (int)del_indexes.size()) continue;
      int slot = del_indexes[i];
      char* pod = records[i];
      m_table->WriteRecordDirect(slot, pod, length);
    }
  }

  //	for (int i = 0; i < valid_count; i++)
  //	{
  //		const char* record = m_table->ReadRecordDirect(i);
  //        BASSERT(record && record[flagoffset] == 0);
  //	}
  m_head.record_count = valid_count;
  m_head.del_count = 0;
  UpdateHead();
  log_info("[DICT]清除标记为待删除的记录成功");
  return true;
}
bool DictTable::CheckFileHeadDirectly(const std::string& fname,
                                      std::string& error) {
  BASSERT(m_head.head_size > 0);
  BASSERT(m_table);

  QFile* file;
  FileSys::instance()->createFile(fname.c_str(), QIODevice::ReadOnly, file);
  if (!file) {
    error = "文件打开失败";
    return false;
  }

  int fileSize = -1;
  fileSize = file->size();
  if (fileSize < m_head.head_size) {
    error = "文件过小";
    return false;
  }

  ScopedArrayPtr<char> headbuf(m_head.head_size + 1);
  if (!file->read((char*)headbuf, m_head.head_size)) {
    error = "文件读取失败";
    return false;
  }

  TableHead* phead = (TableHead*)headbuf.getptr();
  FieldDesc* pdesc = (FieldDesc*)(headbuf.getptr() + sizeof(TableHead));
  // 判断phead,pdesc和m_head,m_record_descs的兼容性
  bool suc = CheckValid(phead, pdesc, error);
  if (suc) {
    log_warning(
        "[DICT]数据检查成功%s,总共%d条记录,%d条被标记为删除,md5:%s  "
        "updateid:%d",
        m_head.name, m_head.record_count, m_head.del_count, m_head.state,
        m_head.updateid);
  } else
    error = "文件头检失败";
  return suc;
}
bool DictTable::CheckFileHead(string& error) {
  // 从映射文件中读取文件头,与内存中的信息进行对比
  // 成功后返回false
  // 失败后设置error信息,返回false;
  BASSERT(m_head.head_size > 0);
  BASSERT(m_table);
  char* headbuf = new char[m_head.head_size];
  memset(headbuf, 0, m_head.head_size);
  m_table->GetCache()->ReadHead(headbuf, m_head.head_size);
  TableHead* phead = (TableHead*)headbuf;
  FieldDesc* pdesc = (FieldDesc*)(headbuf + sizeof(TableHead));
  // 判断phead,pdesc和m_head,m_recoord_descs的兼容性
  bool suc = CheckValid(phead, pdesc, error);
  if (suc) {
    memcpy(&m_head, phead, sizeof(TableHead));
    log_warning(
        "[DICT]数据检查成功%s,总共%d条记录,%d条被标记为删除,md5:%s  "
        "updateid:%d",
        m_head.name, m_head.record_count, m_head.del_count, m_head.state,
        m_head.updateid);
  }
  if (headbuf) delete[] headbuf;
  return suc;
}
RecordId DictTable::Remove(const char* key) {
  const char* dictkey = DictDesc::GetDebugKey();
  if (dictkey && strcmp(key, dictkey) == 0) {
    log_debug("[DICT][DEBUGKEY]key:%s removed", key);
  }
  RecordId rid = m_table->RemoveRecord(key);
  if (rid >= 0) m_head.del_count++;
  return rid;
}

RecordId DictTable::Update(const char* record, int size) {
  BASSERT(m_table && record && ROUND4(size) == m_head.record_size);
  return m_table->WriteRecordDirect(record, size);
}

void DictTable::UpdateHead() {
  // m_head.record_count = m_table->GetRecordCount();
  m_head.crc = 0;  // GetHeadCRC(&m_head);

  int headsize = sizeof(TableHead) + m_record_descs.size() * sizeof(FieldDesc);
  BASSERT(headsize == m_head.head_size);
  char* headbuf = new char[headsize];
  memcpy(headbuf, &m_head, sizeof(TableHead));
  char* bufptr = headbuf + sizeof(TableHead);
  for (int i = 0; i < (int)m_record_descs.size(); i++) {
    memcpy(bufptr, &m_record_descs[i], sizeof(FieldDesc));
    bufptr += sizeof(FieldDesc);
  }
  m_table->GetCache()->WriteHead((const char*)headbuf, headsize,
                                 m_head.record_count);
  delete[] headbuf;
}
void DictTable::ClearSeqSet() { m_seqset.clear(); }
void DictTable::ClearUpdateSet() {
  int nrecord = 0;
  int nmemory = 0;
  int ncount = m_update_map.size();
  m_update_map.Reset(&nrecord);
  nmemory = nrecord * m_pod_record_size;
  log_debug(
      "[DICT] Table:%s memory:%9d  m_update_maphas %d items,%d records %d "
      "bytes ",
      m_head.name, m_table->GetProperty(kTablePropertyMemory), ncount, nrecord,
      nmemory);
}
bool DictTable::GetUpdatedRecord(RecordId rid, char* record, int size) {
  BASSERT(record && size >= GetRecordLength());
  const char* data = m_table->ReadRecord(rid);
  if (data) {
    memcpy(record, data, GetRecordLength());
    return true;
  }
  return false;
}
void DictTable::UpdateMacroIndex(bool sync) {
  BS_METRICS("[DICT] %d", m_macroindex_dict->GetCount());
  BASSERT(m_type == kDictTypeMacroIndex);

  int nold = m_head.record_count;
  std::string md5 = m_macroindex_dict->GetMD5();
  int updateid = m_macroindex_dict->GetUpdateId();

  // 先重置文件头部
  FIELDCOPY(m_head.state, "");
  m_head.updateid = 0;
  m_head.record_count = 0;
  UpdateHead();

  // 更新文件内容;
  int rid = m_macroindex_dict->UpdateTable(m_table, sync);
  log_info("[DICT] maxrid:%d", rid);

  // 写入正确的头部信息
  FIELDCOPY(m_head.state, md5.c_str());
  m_head.updateid = updateid;
  m_head.record_count = rid;
  UpdateHead();
}

namespace {
static bool IsSequenceLogEnable() {
  static bool init = false;
  static bool enable = false;
  if (!init) {
    init = true;
    enable = false;
  }
  return enable;
}
}  // namespace
int DictTable::UpdatePack(UpdatePacket* raw, bool all) {
  const char* buf = raw->fileBuf;
  int len = raw->fileLen;
  const char* md5 = raw->szMd5;
  int updateid = raw->updateId;

  if (IsSequenceLogEnable()) {
    // 用于将字典包保存到本地文件,特定场合检查问题,不影响性能
    static int index = 0;
    if (index == 0) {
      char fpath[MAX_PATH] = {0};
      FMTBUF(fpath, "./Bonddata/msg/%lld",
             QCoreApplication::applicationPid());  // 2019 support lld
      FileSys::instance()->makePathExist(fpath);
    }
    wchar_t fname[MAX_PATH] = {0};
#ifdef _MSC_VER
    FMTBUFW(fname, L"./Bonddata/msg/%lld/%06d_%hs_%hs_%d_%d.msg",
            QCoreApplication::applicationPid(), index++,
            DictDesc::GetStockDictDesc(m_type).GetName().c_str(), md5, updateid,
            len);
#else
    FMTBUFW(fname, L"./Bonddata/msg/%lld/%06d_%s_%s_%d_%d.msg",
            QCoreApplication::applicationPid(), index++,
            DictDesc::GetStockDictDesc(m_type).GetName().c_str(), md5, updateid,
            len);
#endif  // _MSC_VER

    qb::base::StringUtil::SaveFileContent(buf, len, fname);
  }
  /*
  string__string__

  StartPos+8;
  1.开头是类型为PackHeader,大小为14字节的头部
  2.紧接其后的是"string__string__string__"
  序列,表示每个字段的名称,之间间隔2个字符, 指示每条记录中的每个字段的出现的顺序
  3.从nStartPos+8开始,是逐条记录,两条记录之间间隔8字节
  4.每条记录内部的格式:
  字节0	类型,值为SSTYPE_INT32,SSTYPE_STRING,SSTYPE_DOUBLE,STYPE_T32
  字节1	期望是0,如果不为0,那么忽略后续部分
  内容	对于SSTYPE_STRING类型,为strlen(str)+1,其他则为相应sizeof(type)

  存在以下几种数据类型转换,左边类型是kPackDateType,右边类型是kDataType
  除此之外的都是不允许应该报错的
  string-->string
  string-->char
  int-->int
  time32-->time32
  double-->double
  double-->string  %.4f
  double-->string  %.0f

  当all为true,表示首次加载,无需更新m_update_map,获取数据的时候也不使用这个,为了提高性能
  */
  // tag:0x44425253
  // BS_METRICS("");
  qb::base::HighResTime time;
  qb::base::HighResTime field_time(0.0), head_time(0.0);
  PackHeader head = *((PackHeader*)buf);
  std::vector<const char*> fields;
  fields.reserve(head.FieldCount);
  int pack_headsize = FIELD_OFFSET(PackHeader, FieldCount) +
                      RTL_FIELD_SIZE(PackHeader, FieldCount);
  const char* field_start = buf + pack_headsize;
  bool has_sequence = false;
  for (int i = 0; i < head.FieldCount; i++) {
#ifdef WIN32
    if (!has_sequence && _stricmp(field_start, "Sequence") == 0)
#else
    if (!has_sequence && strcasecmp(field_start, "Sequence") == 0)
#endif
      has_sequence = true;
    fields.push_back(field_start);
    field_start += strlen(field_start) + 2;
  }
  const std::vector<FieldDesc>& field_descs = GetRawFieldsByNames(fields);
  BASSERT(field_descs.size() == head.FieldCount);
  // 要做一个特殊处理

  const char* record_start = buf + head.StartPos + 8;
  int typelens[kPackDateTypeCount] = {0,
                                      0,
                                      0,
                                      0,
                                      0,
                                      sizeof(int),
                                      0,
                                      0,
                                      0,
                                      0,
                                      1,
                                      0,
                                      sizeof(double),
                                      0,
                                      sizeof(time_t),
                                      0};  // 各个类型的数据长度,参见_SkipBuffer
  int nfield = field_descs.size();
  int pod_size = GetRecordLength();
  char* pod_record = GetPodRecord(pod_size);
  int record_updated = 0;
  int ndel = 0, nupdate = 0, nignore = 0;

  int seq_min = -1, seq_max = -1, seq_num = 0;  // Sequence字段的值
  int seq_col = -1, seq_offset = -1;
  GetSequenceInfo(seq_col, seq_offset);

  if (0 && m_dict_flag.test(m_type)) {
    log_info("[DICT][DATA]%s UPDATEID:%8d MD5:%s RECORD:%6d  FIELD:%3d",
             DictDesc::GetStockDictDesc(m_type).GetVersion().c_str(), updateid,
             md5, head.RecordCount, head.FieldCount);
  }
  m_table->SetCacheCapcity(head.RecordCount);
  field_time = time.elapsed();
  for (int i = 0; i < (int)head.RecordCount; i++) {
    const char* record = record_start;
    memset(pod_record, 0, pod_size);
    int ignore_col_count = 0;
    for (int j = 0; j < (int)head.FieldCount; j++) {
      kPackDateType type = (kPackDateType)(*record);

      char gap = record[1];  // 类型之后的一个字节
      const char* data = record + 2;  // 变长的数据开始,长度由类型和内容决定
      if (gap != 0) {
        // log_warning("ignore col:%d", j);
        record = record + 2;
        ignore_col_count++;
        continue;
      }

      int len = 2 + typelens[type];
      int dlen = (type == kPackDateTypeString) ? strlen(data) : 0;
      len += dlen;
      if (j >= nfield || field_descs[j].size <= 0)  // 此字段QB没有,忽略
      {
        record = record + len;
        continue;
      }
      const FieldDesc& fd = field_descs[j];
      switch (type) {
        case kPackDateTypeInt32:
          if (fd.type == kDataTypeInt)
            *((int*)(pod_record + fd.offset)) = *(int*)data;
          else {
            LogInvalidField(fd, m_head.name, "Int32");
          }
          break;
        case kPackDateTypeTime32:
          if (fd.type == kDataTypeTime)
            *((time_t*)(pod_record + fd.offset)) = *(time_t*)data;
          else {
            LogInvalidField(fd, m_head.name, "Time32");
          }
          break;
        case kPackDateTypeDouble:
          if (fd.type == kDataTypeDouble)
            *((double*)(pod_record + fd.offset)) = *(double*)data;
          else if (fd.type == kDataTypeCharArray)
            CopyDoubleToField(pod_record + fd.offset, fd.size, "%.4f",
                              *(double*)data);
          else if (fd.type == kDataTypeDate)
            CopyDoubleToField(pod_record + fd.offset, fd.size, "%.0f",
                              *(double*)data);
          else {
            LogInvalidField(fd, m_head.name, "Double");
          }
          break;
        case kPackDateTypeString:
          if (fd.type == kDataTypeCharArray) {
            if (IsAsciiString((const unsigned char*)data, dlen))
              CopyStringToField(pod_record + fd.offset, fd.size, data, dlen);
            else {
              const char* source = m_converter.ToGbk(data, dlen);
              CopyStringToField(pod_record + fd.offset, fd.size, source,
                                m_converter.GetStrSize());
            }
          } else if (fd.type == kDataTypeChar)
            pod_record[fd.offset] = *data;
          else {
            LogInvalidField(fd, m_head.name, "String");
          }
          break;
        default:
          LogInvalidField(fd, m_head.name, "None");
          break;
      }
      record = record + len;
    }
    record_start = record + 8;  // 两条记录之间间隔8字节
    // if (ignore_col_count <= 0)
    {
      // 处理本条记录的序号
      int seq = -1;
      bool seq_exist = false;
      if (seq_offset >= 0 && has_sequence) {
        seq = *((const int*)(pod_record + seq_offset));
        seq_exist = m_seqset.find(seq) != m_seqset.end();
        m_seqset.insert(seq);
        if (seq != 0) seq_num++;
        if (seq_min < 0 || seq_max < 0)
          seq_min = seq_max = seq;
        else {
          seq_min = seq_min > seq ? seq : seq_min;
          seq_max = seq_max < seq ? seq : seq_max;
        }
      }

      if (seq_exist) {
        nignore++;
        record_updated++;
        log_error("[DICT][DATA]由于seq:%d出现相同情况，忽略如下记录:%d", seq,
                  i);
        DictDesc::GetStockDictDesc(m_type).PrintRecord(i, pod_record, pod_size,
                                                       false);
      } else {
        bool removed = UpdatePodRecord(pod_record, pod_size, all);
        if (0 && m_dict_flag.test(m_type) &&
            (seq >= 0 && m_diffset.find(seq) != m_diffset.end())) {
          const DictDesc& dd = DictDesc::GetStockDictDesc(m_type);
          dd.PrintRecord(i, pod_record, pod_size, removed);
          int keyoffset = m_table->GetColumnInfo(m_key_column).GetOffset();
          int rid = m_table->GetRecordIdByKey(pod_record + keyoffset);
          if (removed) {
            BASSERT(rid < 0);
          } else {
            BASSERT(rid >= 0);
          }
        }
        ndel += removed ? 1 : 0;
        nupdate += removed ? 0 : 1;
        record_updated++;
      }
    }
    // else
    //{
    //	log_error("ignore col count:%d", ignore_col_count);
    // }
  }

  // kDictTypeMacroIndex表只更新到内存中,在最后才写入到文件中
  // 将当前包状态和记录个数更新到文件头中
  head_time = qb::base::HighResTime();
  int nold = 0, nnew = 0, nkey = 0;
  if (m_type == kDictTypeMacroIndex) {
    nold = m_macroindex_dict->GetCount();
    m_macroindex_dict->UpdateState(raw->szMd5, raw->updateId);
    nkey = nnew = m_macroindex_dict->GetCount();
  } else {
    // 更新文件头部
    nold = m_head.record_count;
    FIELDCOPY(m_head.state, md5);
    m_head.updateid = updateid;
    m_head.record_count = m_table->GetMaxRecordId() + 1;
    UpdateHead();
    nnew = m_head.record_count;
    nkey = m_table->GetRecordCount();
  }

  head_time = head_time.elapsed();
  double field_time_ratio =
      (double)field_time * 100.0 / ((double)time.elapsed());
  double head_time_ratio = (double)head_time * 100.0 / ((double)time.elapsed());
  unsigned int speed =
      (ndel + nupdate + nignore) == 0
          ? 0
          : (time.elapsed().microsec() / (ndel + nupdate + nignore));
  log_debug(
      "[DICT] process packet %10dus each:%10dus hasseq:%s %s seq:%d "
      "updateid:%d patch:%s md5:%s len:%d origsize:%d  del:%d update:%d "
      "ignore:%d  size:%d-->%d key:%d",
      time.elapsed().microsec(), speed,
      os::fmtstr("%d-[%d:%d-%d]", has_sequence ? 1 : 0, seq_num, seq_min,
                 seq_max)
          .c_str(),
      os::fmtstr("count:%5d/%5d ratio:%.2f/%.2f", head.RecordCount,
                 ndel + nupdate, field_time_ratio, head_time_ratio)
          .c_str(),
      raw->sequence, raw->updateId, raw->szPatchName, raw->szMd5, raw->fileLen,
      raw->origsize, ndel, nupdate, nignore, nold, nnew, nkey);

  return record_updated;
}
bool DictTable::UpdatePodRecord(char* pod_record, int pod_size, bool all) {
#ifdef WIN32
  __try {
#else
  try {
#endif
    return UpdatePodRecordImpl(pod_record, pod_size, all);
  }
#ifdef WIN32
  __except (1)
#else
  catch (...)
#endif
  {
    return false;
  }
  return true;
}
bool DictTable::UpdatePodRecordImpl(char* pod_record, int pod_size, bool all) {
  // 要做一个特殊处理
  enum kProcessStep {
    kProcessStepBeforeUpdate,
    kProcessStepMergeKeyField,
    kProcessStepStringReplace,
    kProcessStepPrintDebugRecord,
    kProcessStepUpdateElement,
    kProcessStepUpdateOrRemove,
    kProcessStepUpdateState,
    kProcessStepFinish,
    kProcessStepCount
  };
  class StepInfo {
    char* m_pod;
    int m_size;
    kDictType m_type;
    int m_count;
    bool m_steps[kProcessStepCount];

   public:
    StepInfo() { SetParam(0, 0, kDictTypeBondList); }
    void SetParam(char* pod, int size, kDictType type) {
      m_count++;
      m_pod = pod;
      m_size = size;
      m_type = type;
      memset(m_steps, 0, sizeof(bool) * kProcessStepCount);
    }
    void SetStep(kProcessStep step) { m_steps[step] = true; }
  };
  // static StepInfo step;

  BASSERT(m_pod_record == pod_record);
  BASSERT(m_pod_record_size == pod_size && pod_size > 0);

  BASSERT(m_table);
  BASSERT(m_key_column >= 0 && m_key_column < m_table->GetColumnCount());
  if (m_keyfield_maker) {
    const ColumnInfo& ci = m_table->GetColumnInfo(m_key_column);
    m_keyfield_maker->BeforeUpdate(pod_record, m_table->GetRecordLength(),
                                   m_head.record_name, ci.GetName(),
                                   ci.GetOffset(), ci.GetFieldBytes());
  }

  BASSERT(m_flag_column >= 0 && m_flag_column < m_table->GetColumnCount());
  int flagoffset = m_table->GetColumnInfo(m_flag_column).GetOffset();
  int keyoffset = m_table->GetColumnInfo(m_key_column).GetOffset();
  bool removed = pod_record[flagoffset] == 'D' ? true : false;

  MergeKeyField(pod_record, m_table->GetRecordLength());
  if (m_type == kDictTypeIssuerInfo) {
    QB_ISSUERINFO* issue = (QB_ISSUERINFO*)pod_record;

    GbkStringReplace(issue->Listed_Type,
                     RTL_FIELD_SIZE(QB_ISSUERINFO, Listed_Type), '|', ',');
  }

  const char* dictkey = DictDesc::GetDebugKey();
  if (dictkey) PrintDebugRecord(dictkey, pod_record, keyoffset, flagoffset, -1);

  RecordId rid = -1;
  if (m_type == kDictTypeMacroIndex) {
    if (!m_macroindex_dict->UpdateElement(*(DICT_MACRO_INDEX*)pod_record)) {
      const DICT_MACRO_INDEX* macro = (const DICT_MACRO_INDEX*)pod_record;
      log_error(
          "[DICT]invalid DICT_MACRO_INDEX Date:%d EndDate:%d Value:%f Code:%s",
          macro->Date, macro->EndDate, macro->Value, macro->Code);
    }
  } else {
    pod_record[flagoffset] = removed;
    if (removed)
      rid = Remove(pod_record + keyoffset);
    else {
      rid = Update(pod_record, GetRecordLength());
    }
  }

  if (!all) {
    AddUpdateSet(pod_record, pod_size, rid, removed, keyoffset);
  }
  return removed;
}
void DictTable::AddUpdateSet(char* pod_record, int pod_size, RecordId rid,
                             bool removed, int keyoffset) {
  RecordState state;
  memset(&state, 0, sizeof(RecordState));
  state.rid = rid;
  state.removed = removed;

  int seq = 0;
  int seq_col = -1, seq_offset = -1;
  if (GetSequenceInfo(seq_col, seq_offset)) {
    seq = *((const int*)(pod_record + seq_offset));
  }
  state.seq = seq;
  state.record = nullptr;
  m_update_map.Add(pod_record, keyoffset, pod_size, state);
}
int DictTable::UpdateModifyPack(const char* key, const char* fieldName,
                                const char* fieldValue) {
  if (key == NULL || fieldValue == NULL) return 0;

  int pod_size = GetRecordLength();
  char* pod_record = GetPodRecord(pod_size);
  int rid = GetRecordId(key);
  int col = GetColumnByName(fieldName);
  if (rid < 0) {
    log_error("[DICT]can not find record with key:%s");
    return 0;
  }
  if (col < 0) {
    log_error("[DICT]can not find column with name:ShortName");
    return 0;
  }
  const char* record = ReadRecord(rid);
  if (record == NULL) {
    log_error("[DICT]ReadRecord(%d) failed", rid);
    return 0;
  }

  const FieldDesc& field = m_record_descs[col];
  memcpy(pod_record, record, GetRecordLength());
  memset(pod_record + field.offset, 0, field.size);

  switch (field.type) {
    case kDataTypeCharArray:
      FIELDCOPYN(pod_record + field.offset, field.size, fieldValue);
      break;
    case kDataTypeChar:
      pod_record[field.offset] = *fieldValue;
      break;
    case kDataTypeInt:
      *((int*)(pod_record + field.offset)) = atoi(fieldValue);
      break;
    case kDataTypeFloat:
      *((float*)(pod_record + field.offset)) = atof(fieldValue);
      break;
    case kDataTypeDouble:
      *((double*)(pod_record + field.offset)) = atof(fieldValue);
      break;
#ifdef WIN32
    case kDataTypeBool:
      *((bool*)(pod_record + field.offset)) =
          _stricmp(fieldValue, "true") == 0 ? true : false;
      break;
#else
    case kDataTypeBool:
      *((bool*)(pod_record + field.offset)) =
          strcasecmp(fieldValue, "true") == 0 ? true : false;
      break;
#endif
    default:
      log_error("UpdateModifyPack 不支持类型为 %s(kDataType) 的字段:%s",
                field.type, fieldName);
      return 0;
      break;
  }
  UpdatePodRecord(pod_record, pod_size, false);
  return 1;
}
RecordId DictTable::GetRecordId(const char* key) {
  BASSERT(key);
  return m_table->GetRecordIdByKey(key);
}
void DictTable::GetAllRecord(vector<unsigned int>& records) {
  m_table->GetCache()->GetAllRecord(records);
}
bool DictTable::ReadRecord(RecordId rid, char* record, int size) {
  BASSERT(rid >= 0 && record && size >= GetRecordLength());
  const char* data = ReadRecord(rid);
  if (data) {
    memcpy(record, data, GetRecordLength());
    return true;
  }
  return false;
}
const char* DictTable::ReadRecord(RecordId rid) {
  // 直接读取原始值,key是合并了的,flag是bool:1表示删除,0表示正常
  /*
          ReadRecordDirect 跳过cache,只把cache当作一个map<key,index>的查找表
          直接使用映射文件来读写
  */
  // const char* data = m_table->ReadRecord(rid);
  const char* data = m_table->ReadRecordDirect(rid);
  return data;
}

char* DictTable::GetPodRecord(int& size) {
  int pod_size = size + 128;
  if (m_pod_record == 0 || m_pod_record_size < pod_size) {
    if (m_pod_record) delete[] m_pod_record;
    m_pod_record = new char[pod_size];
    m_pod_record_size = pod_size;
  }
  memset(m_pod_record, 0, m_pod_record_size);
  size = m_pod_record_size;
  return m_pod_record;
}
void DictTable::FlushCache() { m_table->GetCache()->Flush(); }
void DictTable::DumpStatus(int mode) {
  if (m_table == nullptr) {
    log_debug("[MEMORY][TABLE]table:%s is null", m_head.name);
  } else
    m_table->DumpStatus(mode);
}
bool DictTable::GetExportFields(vector<FieldDesc>& fields) {
  BASSERT(m_type >= 0 && m_type < kDictTypeCount);
  const DictDesc& dd = DictDesc::GetStockDictDesc(m_type);
  const std::vector<int>& indexes = dd.GetExportFields();
  BASSERT(indexes.size() <= m_record_descs.size());
  for (int i = 0; i < (int)indexes.size(); i++) {
    int index = indexes[i];
    FieldDesc& fd = m_record_descs[index];
    if (index == m_flag_column) continue;
    if (index != m_key_column || m_key_split_fields.empty())
      fields.push_back(fd);
    else {
      for (int j = 0; j < (int)m_key_split_fields.size(); j++)
        fields.push_back(m_key_split_fields[j]);
    }
  }
  return true;
}
bool DictTable::GetRawFields(vector<FieldDesc>& fields, bool withFlag) {
  for (int i = 0; i < (int)m_record_descs.size(); i++) {
    FieldDesc& fd = m_record_descs[i];
    if (!withFlag && i == m_flag_column) continue;
    if (i != m_key_column || m_key_split_fields.empty())
      fields.push_back(fd);
    else {
      for (int j = 0; j < (int)m_key_split_fields.size(); j++)
        fields.push_back(m_key_split_fields[j]);
    }
  }
  return true;
}
const std::vector<FieldDesc>& DictTable::GetRawFieldsByNames(
    const std::vector<const char*>& names) {
  // 检查两次的名称是否相同
  // 如果相同,那么继续使用上次的m_last_fields

  bool names_equal = false;
  if (m_last_fields.size() == names.size() &&
      m_last_field_names.size() == names.size()) {
    names_equal = true;
    for (int i = 0; i < (int)names.size(); i++) {
      if (strcmp(m_last_field_names[i].c_str(), names[i]) != 0) {
        names_equal = false;
        break;
      }
    }
  }
  if (!names_equal) {
    m_last_field_names.clear();
    m_last_fields.clear();
    vector<FieldDesc> tempfields;
    GetRawFields(tempfields);
    typedef map<string, int> StringIntMap;
    StringIntMap namemap;
    for (int i = 0; i < (int)tempfields.size(); i++)
      namemap[tempfields[i].name] = i;
    for (int i = 0; i < (int)names.size(); i++) {
      m_last_field_names.push_back(names[i]);
      StringIntMap::iterator it = namemap.find(names[i]);
      if (it == namemap.end()) {
        FieldDesc fd(kDataTypeNone, 0, 0, "", "");
        m_last_fields.push_back(fd);
      } else
        m_last_fields.push_back(tempfields[it->second]);
    }
  }
  return m_last_fields;
}

void DictTable::MergeKeyField(char* record, int length) {
  // 如果存在拼合关键列的情况,那么这里需要将原始列的数据拼凑成一个关键列数据
  BASSERT(record && length == m_table->GetRecordLength());
  if (!m_key_split_fields.empty()) {
    FieldDesc& keyfd = m_record_descs[m_key_column];
    char* data = record + keyfd.offset;
    char* dataend = data + keyfd.size;
    char* ptr = data + strlen(data);
    for (int j = 1; j < (int)m_key_split_fields.size(); j++) {
      FieldDesc& fd = m_key_split_fields[j];
      int fdlen = strlen(record + fd.offset);
      if (fdlen > 0) {
        fdlen = fdlen >= fd.size ? fd.size : fdlen;
        *ptr = m_key_linkchar;
        ptr++;
        memmove(ptr, record + fd.offset, fdlen);
        ptr += fdlen;
      }
    }
    // 清除末尾残留信息
    if (ptr < dataend) memset(ptr, 0, dataend - ptr);
  }
}
void DictTable::SplitKeyField(char* record, int length) {
  BASSERT(record && length == m_table->GetRecordLength());
  if (!m_key_split_fields.empty()) {
    FieldDesc& keyfd = m_record_descs[m_key_column];
    char* data = record + keyfd.offset;

    int datalen = strlen(data);
    char* dataend = data + datalen;
    struct SplitInfo {
      int pos, len;
      SplitInfo() : pos(0), len(0) {}
      SplitInfo(int p, int l) : pos(p), len(l) {}
    };

    // 将主键列内容以m_key_linkchar来分割开
    int tknum = 0;
    SplitInfo tokens[8];
    BASSERT(8 >= m_key_split_fields.size());
    BASSERT(datalen < keyfd.size);
    char* start = data;
    for (int i = 0; i < (int)m_key_split_fields.size(); i++) {
      char* split = strchr(start, m_key_linkchar);
      if (split == 0 || split >= dataend)
        tokens[tknum++] = SplitInfo(start - data, dataend - start);
      else
        tokens[tknum++] = SplitInfo(start - data, split - start);
      start = split + 1;
      if (split == 0 || split >= dataend) break;
    }

    // 从后到前,将以m_key_linkchar分割的关键列数据逐个分离到m_key_split_fields字段对应的位置中
    char* temp = (char*)alloca(length);
    memcpy(temp, record, length);
    for (int i = m_key_split_fields.size() - 1; i >= 0; i--) {
      FieldDesc& fd = m_key_split_fields[i];
      if (i >= tknum) {
        memset(record + fd.offset, 0, fd.size);
        continue;
      }
      int len = tokens[i].len;
      if (len > 0) memmove(record + fd.offset, temp + tokens[i].pos, len);
      if (fd.size - len > 0) memset(record + fd.offset + len, 0, fd.size - len);
    }
  }
}
unsigned int DictTable::GetHeadCRC(const TableHead* head) {
  int size = FIELD_OFFSET(TableHead, crc) + RTL_FIELD_SIZE(TableHead, crc);
  unsigned int crc = qb::base::Base64::CRC32(((const char*)head) + size,
                                             head->head_size - size);
  return crc;
}
bool DictTable::CheckValid(TableHead* head, FieldDesc* pdesc,
                           std::string& error) {
  /*
  head 和pdesc是从映射文件中读取进来的数据,
  m_head和m_record_descs是当前程序的内存中的数据
  这两份数据可能不一致,如QB升级后,增加新列了,列字段字节数变大变小了,
  列类型变了,
  或者 映射文件中本身存在错误

  此函数就是用于判断是否合法且与内存中一致
  如果不一致,那么后续会摧毁此文件内容,然后向服务器全量获取,这是尽量要避免的,除非变更了数据
  此函数不写日志,而是将错误信息放在error中

  有2个字段不做检查:
  state			本地文件的版本状态,类似之前文件的md5里面的值
  record_count	本地文件的记录个数,类似之前BondList.cod的行数
  fast 用于快速返回,此值可改为false;
  */
  bool fast = true;
  bool suc = true;
  do {
#define CHECK_FIELD(P, FIELD)                                         \
  suc = CheckValue(P->FIELD, m_head.FIELD, #FIELD, error, -1) && suc; \
  if (fast && !suc) return false;
    // 检查数据版本信息
    CHECK_FIELD(head, tag);
    // unsigned int crc = GetHeadCRC(head);
    // if (crc != head->crc)
    //{
    //	error = fmtstr("Check crc failed %s.%0x%x!=0x%x", m_head.name,
    // head->crc,crc); 	return false;
    // }

    CHECK_FIELD(head, name);
    CHECK_FIELD(head, version);
    CHECK_FIELD(head, record_name);

    // 检查头部大小,描述偏移量,记录偏移量,描述大小,列个数,单个记录大小
    CHECK_FIELD(head, head_size);
    CHECK_FIELD(head, desc_offset);
    CHECK_FIELD(head, record_offset);
    CHECK_FIELD(head, desc_item_size);
    CHECK_FIELD(head, desc_item_count);
    CHECK_FIELD(head, record_size);
    CHECK_FIELD(head, key_column);
    CHECK_FIELD(head, flag_column);
    CHECK_FIELD(head, view_size);
    CHECK_FIELD(head, inc_size);

    for (int i = 0; i < m_head.desc_item_count; i++) {
      FieldDesc& fd = m_record_descs[i];
      suc = CheckValue(fd.type, pdesc[i].type, "type", error, i) && suc;
      if (fast && !suc) return false;
      suc = CheckValue(fd.offset, pdesc[i].offset, "offset", error, i) && suc;
      if (fast && !suc) return false;
      suc = CheckValue(fd.name, pdesc[i].name, "name", error, i) && suc;
      if (fast && !suc) return false;
      suc = CheckValue(fd.desc, pdesc[i].desc, "desc", error, i) && suc;
      if (fast && !suc) return false;
    }
  } while (false);
  return suc;
}
bool DictTable::CheckValue(int n1, int n2, const char* name, std::string& error,
                           int field) {
  if (n1 != n2) {
    if (field < 0)
      error += os::fmtstr("Check %s.%s:%d!=%d", m_head.name, name, n1, n2);
    else
      error += os::fmtstr("Check %s.%s[%d]:%d!=%d", m_head.name, name, field,
                          n1, n2);
  }
  return n1 == n2;
}
bool DictTable::CheckValue(const char* s1, const char* s2, const char* name,
                           std::string& error, int field) {
  if (strcmp(s1, s2) != 0) {
    if (field < 0)
      error += os::fmtstr("Check %s.%s:%s!=%s", m_head.name, name, s1, s2);
    else
      error += os::fmtstr("Check %s.%s[%d]:%s!=%s", m_head.name, name, field,
                          s1, s2);
    return false;
  }
  return true;
}

FieldDesc::FieldDesc(kDataType t, int off, int sz, const char* n, const char* d)
    : type(t), offset(off), size(sz) {
  memset(name, 0, sizeof(name));
  memset(desc, 0, sizeof(desc));
  FIELDCOPY(name, n);
  FIELDCOPY(desc, d);
}

void DictDesc::AddKeyField(const FieldDesc& fd) {
  const char* dotstr = strchr(fd.name, '.');
  int dot = dotstr ? (dotstr - fd.name) : (-1);
  if (dot < 0)
    m_keyfields.push_back(fd);
  else {
    FieldDesc fd2 = fd;
    memset(fd2.name, 0, RTL_FIELD_SIZE(FieldDesc, name));
    FIELDCOPY(fd2.name, fd.name + dot + 1);
    m_keyfields.push_back(fd2);
  }
}

void DictTable::UpdateSet::Add(const char* podRecord, int keyoffset,
                               int podSize, RecordState& state) {
  char* record = nullptr;
  string key(podRecord + keyoffset);
  RecordStateMap::iterator it = m_statemap.find(key);
  if (it != m_statemap.end()) {
    record = it->second.record;
    m_statemap.erase(it);
  }
  if (state.removed) {
    state.record = record ? record : (new (std::nothrow) char[podSize]);
    record = nullptr;
    if (state.record) memcpy(state.record, podRecord, podSize);
  }
  m_statemap[key] = state;
  if (record) delete[] record;
}

void DictTable::UpdateSet::Reset(int* pnrecord) {
  int nrecord = 0;
  int ncount = size();
  for (RecordStateMap::iterator it = m_statemap.begin(); it != m_statemap.end();
       ++it) {
    RecordState& rs = it->second;
    if (rs.record) {
      nrecord++;
      delete[] rs.record;
      rs.record = NULL;
    }
  }
  if (pnrecord) {
    *pnrecord = nrecord;
  }
  m_statemap.clear();
}

}  // namespace ht
}  // namespace qb
